{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iEkm_Q9AGgTX"
      },
      "source": [
        "# MIXTRAL 8x7B - Mixture of Experts\n",
        "\n",
        "This will not run on the free T4 GPU from Google Colab. You will need A100 to run this."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AlVGJXwsyVQO"
      },
      "source": [
        "### Install Required Packages"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "3IovaXt4ZJQ_"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[33mDEPRECATION: omegaconf 2.0.6 has a non-standard dependency specifier PyYAML>=5.1.*. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of omegaconf or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063\u001b[0m\u001b[33m\n",
            "\u001b[0m\u001b[33mDEPRECATION: omegaconf 2.0.6 has a non-standard dependency specifier PyYAML>=5.1.*. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of omegaconf or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063\u001b[0m\u001b[33m\n",
            "\u001b[0m\u001b[33mDEPRECATION: omegaconf 2.0.6 has a non-standard dependency specifier PyYAML>=5.1.*. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of omegaconf or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063\u001b[0m\u001b[33m\n",
            "\u001b[0m\u001b[33mDEPRECATION: omegaconf 2.0.6 has a non-standard dependency specifier PyYAML>=5.1.*. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of omegaconf or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063\u001b[0m\u001b[33m\n",
            "\u001b[0m\u001b[33mDEPRECATION: omegaconf 2.0.6 has a non-standard dependency specifier PyYAML>=5.1.*. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of omegaconf or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063\u001b[0m\u001b[33m\n",
            "\u001b[0m\u001b[33mDEPRECATION: omegaconf 2.0.6 has a non-standard dependency specifier PyYAML>=5.1.*. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of omegaconf or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063\u001b[0m\u001b[33m\n",
            "\u001b[0mRequirement already satisfied: flash-attn in /home/adithya/miniconda3/envs/train-venv/lib/python3.10/site-packages (2.4.2)\n",
            "Requirement already satisfied: torch in /home/adithya/miniconda3/envs/train-venv/lib/python3.10/site-packages (from flash-attn) (2.1.2+cu121)\n",
            "Requirement already satisfied: einops in /home/adithya/miniconda3/envs/train-venv/lib/python3.10/site-packages (from flash-attn) (0.7.0)\n",
            "Requirement already satisfied: packaging in /home/adithya/miniconda3/envs/train-venv/lib/python3.10/site-packages (from flash-attn) (23.2)\n",
            "Requirement already satisfied: ninja in /home/adithya/miniconda3/envs/train-venv/lib/python3.10/site-packages (from flash-attn) (1.11.1.1)\n",
            "Requirement already satisfied: filelock in /home/adithya/miniconda3/envs/train-venv/lib/python3.10/site-packages (from torch->flash-attn) (3.9.0)\n",
            "Requirement already satisfied: typing-extensions in /home/adithya/miniconda3/envs/train-venv/lib/python3.10/site-packages (from torch->flash-attn) (4.9.0)\n",
            "Requirement already satisfied: sympy in /home/adithya/miniconda3/envs/train-venv/lib/python3.10/site-packages (from torch->flash-attn) (1.12)\n",
            "Requirement already satisfied: networkx in /home/adithya/miniconda3/envs/train-venv/lib/python3.10/site-packages (from torch->flash-attn) (3.0)\n",
            "Requirement already satisfied: jinja2 in /home/adithya/miniconda3/envs/train-venv/lib/python3.10/site-packages (from torch->flash-attn) (3.1.2)\n",
            "Requirement already satisfied: fsspec in /home/adithya/miniconda3/envs/train-venv/lib/python3.10/site-packages (from torch->flash-attn) (2023.10.0)\n",
            "Requirement already satisfied: triton==2.1.0 in /home/adithya/miniconda3/envs/train-venv/lib/python3.10/site-packages (from torch->flash-attn) (2.1.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /home/adithya/miniconda3/envs/train-venv/lib/python3.10/site-packages (from jinja2->torch->flash-attn) (2.1.3)\n",
            "Requirement already satisfied: mpmath>=0.19 in /home/adithya/miniconda3/envs/train-venv/lib/python3.10/site-packages (from sympy->torch->flash-attn) (1.3.0)\n",
            "\u001b[33mDEPRECATION: omegaconf 2.0.6 has a non-standard dependency specifier PyYAML>=5.1.*. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of omegaconf or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063\u001b[0m\u001b[33m\n",
            "\u001b[0m"
          ]
        }
      ],
      "source": [
        "!pip install -q -U bitsandbytes\n",
        "!pip install -q -U git+https://github.com/huggingface/transformers.git\n",
        "!pip install -q -U git+https://github.com/huggingface/peft.git\n",
        "!pip install -q -U git+https://github.com/huggingface/accelerate.git\n",
        "!pip install -q datasets scipy\n",
        "!pip install -q trl\n",
        "!pip install flash-attn --no-build-isolation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4qjy9PYUaY3W"
      },
      "source": [
        "### Loading the Base Model\n",
        "\n",
        "Load the model in `4bit`, with double quantization, with `bfloat16` as the compute dtype.\n",
        "\n",
        "In this case we are using the instruct-tuned model - instead of the base model. For fine-tuning a base model will need a lot more data!"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Load dataset for finetuning"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Lets Load the Dataset\n",
        "\n",
        "For this tutorial, we will fine-tune Mistral 7B Instruct for code generation.\n",
        "\n",
        "We will be using this [dataset](https://huggingface.co/datasets/TokenBender/code_instructions_122k_alpaca_style) which is curated by [TokenBender (e/xperiments)](https://twitter.com/4evaBehindSOTA) and is an excellent data source for fine-tuning models for code generation. It follows the alpaca style of instructions, which is an excellent starting point for this task. The dataset structure should resemble the following:\n",
        "\n",
        "```json\n",
        "{\n",
        "  \"instruction\": \"Create a function to calculate the sum of a sequence of integers.\",\n",
        "  \"input\": \"[1, 2, 3, 4, 5]\",\n",
        "  \"output\": \"# Python code def sum_sequence(sequence): sum = 0 for num in sequence: sum += num return sum\"\n",
        "}\n",
        "```"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {},
      "outputs": [],
      "source": [
        "model_id = \"mistralai/Mixtral-8x7B-v0.1\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {},
      "outputs": [],
      "source": [
        "import torch\n",
        "from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig\n",
        "\n",
        "nf4_config = BitsAndBytesConfig(\n",
        "   load_in_4bit=True,\n",
        "   bnb_4bit_quant_type=\"nf4\",\n",
        "   bnb_4bit_use_double_quant=True,\n",
        "   bnb_4bit_compute_dtype=torch.bfloat16\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0d62f226ec104f04baa1b78a3b7cd10f",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Loading checkpoint shards:   0%|          | 0/19 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "model = AutoModelForCausalLM.from_pretrained(\n",
        "    model_id,\n",
        "    device_map='auto',\n",
        "    quantization_config=nf4_config,\n",
        "    use_cache=False,\n",
        "    attn_implementation=\"flash_attention_2\"\n",
        "\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {},
      "outputs": [],
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(model_id)\n",
        "\n",
        "tokenizer.pad_token = tokenizer.eos_token\n",
        "tokenizer.padding_side = \"right\""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Let's example how well the model does at this task currently:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {},
      "outputs": [],
      "source": [
        "def generate_response(prompt, model):\n",
        "  encoded_input = tokenizer(prompt,  return_tensors=\"pt\", add_special_tokens=True)\n",
        "  model_inputs = encoded_input.to('cuda')\n",
        "\n",
        "  generated_ids = model.generate(**model_inputs,\n",
        "                                 max_new_tokens=512,\n",
        "                                 do_sample=True,\n",
        "                                 pad_token_id=tokenizer.eos_token_id)\n",
        "\n",
        "  decoded_output = tokenizer.batch_decode(generated_ids)\n",
        "\n",
        "  return decoded_output[0].replace(prompt, \"\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "prompt=\"\"\"[INST]Use the provided input to create an instruction that could have been used to generate the response with an LLM. \\nThere are more than 12,000 species of grass. The most common is Kentucky Bluegrass, because it grows quickly, easily, and is soft to the touch. Rygrass is shiny and bright green colored. Fescues are dark green and shiny. Bermuda grass is harder but can grow in drier soil.[\\INST]\"\"\"\n",
        "\n",
        "generate_response(prompt, model)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "MixtralForCausalLM(\n",
            "  (model): MixtralModel(\n",
            "    (embed_tokens): Embedding(32000, 4096)\n",
            "    (layers): ModuleList(\n",
            "      (0-31): 32 x MixtralDecoderLayer(\n",
            "        (self_attn): MixtralFlashAttention2(\n",
            "          (q_proj): Linear4bit(in_features=4096, out_features=4096, bias=False)\n",
            "          (k_proj): Linear4bit(in_features=4096, out_features=1024, bias=False)\n",
            "          (v_proj): Linear4bit(in_features=4096, out_features=1024, bias=False)\n",
            "          (o_proj): Linear4bit(in_features=4096, out_features=4096, bias=False)\n",
            "          (rotary_emb): MixtralRotaryEmbedding()\n",
            "        )\n",
            "        (block_sparse_moe): MixtralSparseMoeBlock(\n",
            "          (gate): Linear4bit(in_features=4096, out_features=8, bias=False)\n",
            "          (experts): ModuleList(\n",
            "            (0-7): 8 x MixtralBLockSparseTop2MLP(\n",
            "              (w1): Linear4bit(in_features=4096, out_features=14336, bias=False)\n",
            "              (w2): Linear4bit(in_features=14336, out_features=4096, bias=False)\n",
            "              (w3): Linear4bit(in_features=4096, out_features=14336, bias=False)\n",
            "              (act_fn): SiLU()\n",
            "            )\n",
            "          )\n",
            "        )\n",
            "        (input_layernorm): MixtralRMSNorm()\n",
            "        (post_attention_layernorm): MixtralRMSNorm()\n",
            "      )\n",
            "    )\n",
            "    (norm): MixtralRMSNorm()\n",
            "  )\n",
            "  (lm_head): Linear(in_features=4096, out_features=32000, bias=False)\n",
            ")\n"
          ]
        }
      ],
      "source": [
        "print(model)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Dataset({\n",
              "    features: ['instruction', 'output', 'input', 'text'],\n",
              "    num_rows: 121959\n",
              "})"
            ]
          },
          "execution_count": 17,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "from datasets import load_dataset\n",
        "\n",
        "dataset = load_dataset(\"TokenBender/code_instructions_122k_alpaca_style\", split=\"train\")\n",
        "dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>instruction</th>\n",
              "      <th>output</th>\n",
              "      <th>input</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Create a function to calculate the sum of a se...</td>\n",
              "      <td># Python code\\ndef sum_sequence(sequence):\\n  ...</td>\n",
              "      <td>[1, 2, 3, 4, 5]</td>\n",
              "      <td>Below is an instruction that describes a task....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Develop a function that will add two strings</td>\n",
              "      <td>def add_strings(str1, str2):\\n    \"\"\"This func...</td>\n",
              "      <td>str1 = \"Hello \"\\nstr2 = \"world\"</td>\n",
              "      <td>Below is an instruction that describes a task....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Design a data structure in C++ to store inform...</td>\n",
              "      <td>#include &lt;map&gt;\\n#include &lt;string&gt;\\n\\nclass Gro...</td>\n",
              "      <td></td>\n",
              "      <td>Below is an instruction that describes a task....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Implement a sorting algorithm to sort a given ...</td>\n",
              "      <td>def bubble_sort(arr):\\n    n = len(arr)\\n \\n  ...</td>\n",
              "      <td>[3, 1, 4, 5, 9, 0]</td>\n",
              "      <td>Below is an instruction that describes a task....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Design a Swift application for tracking expens...</td>\n",
              "      <td>import UIKit\\n\\nclass ExpenseViewController: U...</td>\n",
              "      <td>Not applicable</td>\n",
              "      <td>Below is an instruction that describes a task....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>Create a REST API to convert a UNIX timestamp ...</td>\n",
              "      <td>&lt;?php\\n$timestamp = $_GET['timestamp'];\\n\\nif(...</td>\n",
              "      <td>Not Applicable</td>\n",
              "      <td>Below is an instruction that describes a task....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>Generate a Python code for crawling a website ...</td>\n",
              "      <td>import requests\\nimport re\\n\\ndef crawl_websit...</td>\n",
              "      <td>website: www.example.com \\ndata to crawl: phon...</td>\n",
              "      <td>Below is an instruction that describes a task....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>Create a Python list comprehension to get the ...</td>\n",
              "      <td>[x*x for x in [1, 2, 3, 5, 8, 13]]</td>\n",
              "      <td></td>\n",
              "      <td>Below is an instruction that describes a task....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>Create a MySQL query to find the most expensiv...</td>\n",
              "      <td>SELECT * FROM products ORDER BY price DESC LIM...</td>\n",
              "      <td></td>\n",
              "      <td>Below is an instruction that describes a task....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>Create a data structure in Java for storing an...</td>\n",
              "      <td>public class Library {\\n \\n // map of books in...</td>\n",
              "      <td>Not applicable</td>\n",
              "      <td>Below is an instruction that describes a task....</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                         instruction  \\\n",
              "0  Create a function to calculate the sum of a se...   \n",
              "1       Develop a function that will add two strings   \n",
              "2  Design a data structure in C++ to store inform...   \n",
              "3  Implement a sorting algorithm to sort a given ...   \n",
              "4  Design a Swift application for tracking expens...   \n",
              "5  Create a REST API to convert a UNIX timestamp ...   \n",
              "6  Generate a Python code for crawling a website ...   \n",
              "7  Create a Python list comprehension to get the ...   \n",
              "8  Create a MySQL query to find the most expensiv...   \n",
              "9  Create a data structure in Java for storing an...   \n",
              "\n",
              "                                              output  \\\n",
              "0  # Python code\\ndef sum_sequence(sequence):\\n  ...   \n",
              "1  def add_strings(str1, str2):\\n    \"\"\"This func...   \n",
              "2  #include <map>\\n#include <string>\\n\\nclass Gro...   \n",
              "3  def bubble_sort(arr):\\n    n = len(arr)\\n \\n  ...   \n",
              "4  import UIKit\\n\\nclass ExpenseViewController: U...   \n",
              "5  <?php\\n$timestamp = $_GET['timestamp'];\\n\\nif(...   \n",
              "6  import requests\\nimport re\\n\\ndef crawl_websit...   \n",
              "7                 [x*x for x in [1, 2, 3, 5, 8, 13]]   \n",
              "8  SELECT * FROM products ORDER BY price DESC LIM...   \n",
              "9  public class Library {\\n \\n // map of books in...   \n",
              "\n",
              "                                               input  \\\n",
              "0                                    [1, 2, 3, 4, 5]   \n",
              "1                    str1 = \"Hello \"\\nstr2 = \"world\"   \n",
              "2                                                      \n",
              "3                                 [3, 1, 4, 5, 9, 0]   \n",
              "4                                     Not applicable   \n",
              "5                                     Not Applicable   \n",
              "6  website: www.example.com \\ndata to crawl: phon...   \n",
              "7                                                      \n",
              "8                                                      \n",
              "9                                     Not applicable   \n",
              "\n",
              "                                                text  \n",
              "0  Below is an instruction that describes a task....  \n",
              "1  Below is an instruction that describes a task....  \n",
              "2  Below is an instruction that describes a task....  \n",
              "3  Below is an instruction that describes a task....  \n",
              "4  Below is an instruction that describes a task....  \n",
              "5  Below is an instruction that describes a task....  \n",
              "6  Below is an instruction that describes a task....  \n",
              "7  Below is an instruction that describes a task....  \n",
              "8  Below is an instruction that describes a task....  \n",
              "9  Below is an instruction that describes a task....  "
            ]
          },
          "execution_count": 18,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df = dataset.to_pandas()\n",
        "df.head(10)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Instruction Fintuning - Prepare the dataset under the format of \"prompt\" so the model can better understand :\n",
        "1. the function generate_prompt : take the instruction and output and generate a prompt\n",
        "2. shuffle the dataset\n",
        "3. tokenizer the dataset"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Formatting the Dataset\n",
        "\n",
        "Now, let's format the dataset in the required [Mistral-7B-Instruct-v0.1 format](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1).\n",
        "\n",
        "> Many tutorials and blogs skip over this part, but I feel this is a really important step.\n",
        "\n",
        "We'll put each instruction and input pair between `[INST]` and `[/INST]` output after that, like this:\n",
        "\n",
        "```\n",
        "<s>[INST] What is your favorite condiment? [/INST]\n",
        "Well, I'm quite partial to a good squeeze of fresh lemon juice. It adds just the right amount of zesty flavor to whatever I'm cooking up in the kitchen!</s>\n",
        "```\n",
        "\n",
        "You can use the following code to process your dataset and create a JSONL file in the correct format:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {},
      "outputs": [],
      "source": [
        "def generate_prompt(data_point):\n",
        "    \"\"\"Gen. input text based on a prompt, task instruction, (context info.), and answer\n",
        "\n",
        "    :param data_point: dict: Data point\n",
        "    :return: dict: tokenzed prompt\n",
        "    \"\"\"\n",
        "    prefix_text = 'Below is an instruction that describes a task. Write a response that ' \\\n",
        "               'appropriately completes the request.\\n\\n'\n",
        "    # Samples with additional context into.\n",
        "    if data_point['input']:\n",
        "        text = f\"\"\"<s>[INST]{prefix_text} {data_point[\"instruction\"]} here are the inputs {data_point[\"input\"]} [/INST]{data_point[\"output\"]}</s>\"\"\"\n",
        "    # Without\n",
        "    else:\n",
        "        text = f\"\"\"<s>[INST]{prefix_text} {data_point[\"instruction\"]} [/INST]{data_point[\"output\"]} </s>\"\"\"\n",
        "    return text\n",
        "\n",
        "# add the \"prompt\" column in the dataset\n",
        "text_column = [generate_prompt(data_point) for data_point in dataset]\n",
        "dataset = dataset.add_column(\"prompt\", text_column)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0ff969275a994e9db2f88c90ec8c778e",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Map:   0%|          | 0/121959 [00:00<?, ? examples/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "dataset = dataset.shuffle(seed=1234)  # Shuffle dataset here\n",
        "dataset = dataset.map(lambda samples: tokenizer(samples[\"prompt\"]), batched=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {},
      "outputs": [],
      "source": [
        "dataset = dataset.train_test_split(test_size=0.2)\n",
        "train_data = dataset[\"train\"]\n",
        "test_data = dataset[\"test\"]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Dataset({\n",
              "    features: ['instruction', 'output', 'input', 'text', 'prompt', 'input_ids', 'attention_mask'],\n",
              "    num_rows: 97567\n",
              "})"
            ]
          },
          "execution_count": 25,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "train_data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[[1,\n",
              "  1,\n",
              "  733,\n",
              "  16289,\n",
              "  28793,\n",
              "  20548,\n",
              "  336,\n",
              "  349,\n",
              "  396,\n",
              "  13126,\n",
              "  369,\n",
              "  13966,\n",
              "  264,\n",
              "  3638,\n",
              "  28723,\n",
              "  12018,\n",
              "  264,\n",
              "  2899,\n",
              "  369,\n",
              "  6582,\n",
              "  1999,\n",
              "  2691,\n",
              "  274,\n",
              "  272,\n",
              "  2159,\n",
              "  28723,\n",
              "  13,\n",
              "  13,\n",
              "  12018,\n",
              "  264,\n",
              "  908,\n",
              "  298,\n",
              "  9051,\n",
              "  272,\n",
              "  1679,\n",
              "  2318,\n",
              "  297,\n",
              "  264,\n",
              "  2039,\n",
              "  302,\n",
              "  261,\n",
              "  294,\n",
              "  28733,\n",
              "  28707,\n",
              "  323,\n",
              "  28733,\n",
              "  532,\n",
              "  28706,\n",
              "  28723,\n",
              "  1236,\n",
              "  460,\n",
              "  272,\n",
              "  14391,\n",
              "  5113,\n",
              "  327,\n",
              "  733,\n",
              "  1410,\n",
              "  28814,\n",
              "  647,\n",
              "  464,\n",
              "  28814,\n",
              "  647,\n",
              "  464,\n",
              "  28762,\n",
              "  5807,\n",
              "  5936,\n",
              "  28762,\n",
              "  647,\n",
              "  464,\n",
              "  28814,\n",
              "  647,\n",
              "  464,\n",
              "  28762,\n",
              "  5807,\n",
              "  5936,\n",
              "  28762,\n",
              "  647,\n",
              "  9815,\n",
              "  464,\n",
              "  1421,\n",
              "  28793,\n",
              "  733,\n",
              "  28748,\n",
              "  16289,\n",
              "  28793,\n",
              "  1270,\n",
              "  625,\n",
              "  28730,\n",
              "  3423,\n",
              "  28730,\n",
              "  7125,\n",
              "  28732,\n",
              "  3516,\n",
              "  1329,\n",
              "  28705,\n",
              "  13,\n",
              "  2287,\n",
              "  354,\n",
              "  613,\n",
              "  297,\n",
              "  2819,\n",
              "  28732,\n",
              "  28770,\n",
              "  1329,\n",
              "  28705,\n",
              "  13,\n",
              "  5390,\n",
              "  354,\n",
              "  461,\n",
              "  297,\n",
              "  2819,\n",
              "  28732,\n",
              "  28770,\n",
              "  1329,\n",
              "  28705,\n",
              "  13,\n",
              "  17422,\n",
              "  513,\n",
              "  5113,\n",
              "  28792,\n",
              "  28710,\n",
              "  3328,\n",
              "  28768,\n",
              "  28793,\n",
              "  859,\n",
              "  464,\n",
              "  1869,\n",
              "  28705,\n",
              "  13,\n",
              "  1417,\n",
              "  28705,\n",
              "  604,\n",
              "  613,\n",
              "  28725,\n",
              "  461,\n",
              "  28705,\n",
              "  13,\n",
              "  13,\n",
              "  7125,\n",
              "  28730,\n",
              "  28710,\n",
              "  28725,\n",
              "  2318,\n",
              "  28730,\n",
              "  28768,\n",
              "  327,\n",
              "  625,\n",
              "  28730,\n",
              "  3423,\n",
              "  28730,\n",
              "  7125,\n",
              "  28732,\n",
              "  3516,\n",
              "  28731,\n",
              "  2],\n",
              " [1,\n",
              "  1,\n",
              "  733,\n",
              "  16289,\n",
              "  28793,\n",
              "  20548,\n",
              "  336,\n",
              "  349,\n",
              "  396,\n",
              "  13126,\n",
              "  369,\n",
              "  13966,\n",
              "  264,\n",
              "  3638,\n",
              "  28723,\n",
              "  12018,\n",
              "  264,\n",
              "  2899,\n",
              "  369,\n",
              "  6582,\n",
              "  1999,\n",
              "  2691,\n",
              "  274,\n",
              "  272,\n",
              "  2159,\n",
              "  28723,\n",
              "  13,\n",
              "  13,\n",
              "  5670,\n",
              "  264,\n",
              "  17922,\n",
              "  15733,\n",
              "  369,\n",
              "  4435,\n",
              "  272,\n",
              "  3181,\n",
              "  302,\n",
              "  264,\n",
              "  2901,\n",
              "  477,\n",
              "  2687,\n",
              "  298,\n",
              "  3075,\n",
              "  739,\n",
              "  18848,\n",
              "  286,\n",
              "  754,\n",
              "  28723,\n",
              "  1236,\n",
              "  460,\n",
              "  272,\n",
              "  14391,\n",
              "  1770,\n",
              "  2787,\n",
              "  28723,\n",
              "  733,\n",
              "  28748,\n",
              "  16289,\n",
              "  28793,\n",
              "  1538,\n",
              "  371,\n",
              "  13,\n",
              "  2287,\n",
              "  4850,\n",
              "  28747,\n",
              "  28705,\n",
              "  28750,\n",
              "  28734,\n",
              "  28734,\n",
              "  4125,\n",
              "  28745,\n",
              "  13,\n",
              "  2287,\n",
              "  5110,\n",
              "  28747,\n",
              "  28705,\n",
              "  28750,\n",
              "  28734,\n",
              "  28734,\n",
              "  4125,\n",
              "  28745,\n",
              "  13,\n",
              "  2287,\n",
              "  5414,\n",
              "  28733,\n",
              "  3456,\n",
              "  28747,\n",
              "  2687,\n",
              "  28745,\n",
              "  13,\n",
              "  2287,\n",
              "  8265,\n",
              "  28747,\n",
              "  5414,\n",
              "  28733,\n",
              "  3456,\n",
              "  28705,\n",
              "  28740,\n",
              "  28713,\n",
              "  28745,\n",
              "  13,\n",
              "  28752,\n",
              "  13,\n",
              "  13,\n",
              "  1538,\n",
              "  28747,\n",
              "  20635,\n",
              "  371,\n",
              "  13,\n",
              "  2287,\n",
              "  5414,\n",
              "  28733,\n",
              "  3456,\n",
              "  28747,\n",
              "  3075,\n",
              "  28745,\n",
              "  13,\n",
              "  260,\n",
              "  13,\n",
              "  28752,\n",
              "  2],\n",
              " [1,\n",
              "  1,\n",
              "  733,\n",
              "  16289,\n",
              "  28793,\n",
              "  20548,\n",
              "  336,\n",
              "  349,\n",
              "  396,\n",
              "  13126,\n",
              "  369,\n",
              "  13966,\n",
              "  264,\n",
              "  3638,\n",
              "  28723,\n",
              "  12018,\n",
              "  264,\n",
              "  2899,\n",
              "  369,\n",
              "  6582,\n",
              "  1999,\n",
              "  2691,\n",
              "  274,\n",
              "  272,\n",
              "  2159,\n",
              "  28723,\n",
              "  13,\n",
              "  13,\n",
              "  5670,\n",
              "  264,\n",
              "  11639,\n",
              "  2007,\n",
              "  298,\n",
              "  6603,\n",
              "  264,\n",
              "  1274,\n",
              "  302,\n",
              "  11272,\n",
              "  298,\n",
              "  264,\n",
              "  1274,\n",
              "  302,\n",
              "  3113,\n",
              "  7850,\n",
              "  28723,\n",
              "  1236,\n",
              "  460,\n",
              "  272,\n",
              "  14391,\n",
              "  733,\n",
              "  28835,\n",
              "  28740,\n",
              "  9350,\n",
              "  981,\n",
              "  28750,\n",
              "  9350,\n",
              "  981,\n",
              "  28770,\n",
              "  28838,\n",
              "  28793,\n",
              "  733,\n",
              "  28748,\n",
              "  16289,\n",
              "  28793,\n",
              "  7841,\n",
              "  3070,\n",
              "  28723,\n",
              "  1858,\n",
              "  28723,\n",
              "  23739,\n",
              "  28745,\n",
              "  13,\n",
              "  7841,\n",
              "  3070,\n",
              "  28723,\n",
              "  1858,\n",
              "  28723,\n",
              "  1245,\n",
              "  28745,\n",
              "  13,\n",
              "  7841,\n",
              "  3070,\n",
              "  28723,\n",
              "  1858,\n",
              "  28723,\n",
              "  3888,\n",
              "  28723,\n",
              "  27991,\n",
              "  734,\n",
              "  28745,\n",
              "  13,\n",
              "  13,\n",
              "  2001,\n",
              "  875,\n",
              "  1677,\n",
              "  1551,\n",
              "  7396,\n",
              "  1245,\n",
              "  371,\n",
              "  13,\n",
              "  798,\n",
              "  1062,\n",
              "  1066,\n",
              "  2191,\n",
              "  28732,\n",
              "  1154,\n",
              "  2002,\n",
              "  4707,\n",
              "  28731,\n",
              "  371,\n",
              "  13,\n",
              "  28705,\n",
              "  3231,\n",
              "  28789,\n",
              "  1154,\n",
              "  28767,\n",
              "  11272,\n",
              "  327,\n",
              "  20037,\n",
              "  748,\n",
              "  28723,\n",
              "  293,\n",
              "  1245,\n",
              "  618,\n",
              "  28740,\n",
              "  548,\n",
              "  345,\n",
              "  28750,\n",
              "  548,\n",
              "  345,\n",
              "  28770,\n",
              "  1041,\n",
              "  13,\n",
              "  28705,\n",
              "  3231,\n",
              "  28789,\n",
              "  7396,\n",
              "  28767,\n",
              "  3113,\n",
              "  7850,\n",
              "  327,\n",
              "  11272,\n",
              "  28723,\n",
              "  3888,\n",
              "  1546,\n",
              "  2022,\n",
              "  28732,\n",
              "  7396,\n",
              "  564,\n",
              "  25249,\n",
              "  609,\n",
              "  16077,\n",
              "  28732,\n",
              "  27991,\n",
              "  734,\n",
              "  28723,\n",
              "  532,\n",
              "  1245,\n",
              "  1657,\n",
              "  13,\n",
              "  28705,\n",
              "  2135,\n",
              "  28723,\n",
              "  406,\n",
              "  28723,\n",
              "  7561,\n",
              "  28732,\n",
              "  15883,\n",
              "  7850,\n",
              "  344,\n",
              "  589,\n",
              "  13281,\n",
              "  28713,\n",
              "  733,\n",
              "  28740,\n",
              "  28725,\n",
              "  28705,\n",
              "  28750,\n",
              "  28725,\n",
              "  28705,\n",
              "  28770,\n",
              "  28793,\n",
              "  13,\n",
              "  28705,\n",
              "  443,\n",
              "  13,\n",
              "  28752,\n",
              "  2],\n",
              " [1,\n",
              "  1,\n",
              "  733,\n",
              "  16289,\n",
              "  28793,\n",
              "  20548,\n",
              "  336,\n",
              "  349,\n",
              "  396,\n",
              "  13126,\n",
              "  369,\n",
              "  13966,\n",
              "  264,\n",
              "  3638,\n",
              "  28723,\n",
              "  12018,\n",
              "  264,\n",
              "  2899,\n",
              "  369,\n",
              "  6582,\n",
              "  1999,\n",
              "  2691,\n",
              "  274,\n",
              "  272,\n",
              "  2159,\n",
              "  28723,\n",
              "  13,\n",
              "  13,\n",
              "  26075,\n",
              "  396,\n",
              "  18586,\n",
              "  2700,\n",
              "  297,\n",
              "  21366,\n",
              "  298,\n",
              "  26518,\n",
              "  1318,\n",
              "  28815,\n",
              "  28750,\n",
              "  648,\n",
              "  337,\n",
              "  28815,\n",
              "  28750,\n",
              "  3817,\n",
              "  298,\n",
              "  272,\n",
              "  17656,\n",
              "  1318,\n",
              "  648,\n",
              "  337,\n",
              "  327,\n",
              "  28705,\n",
              "  28740,\n",
              "  28723,\n",
              "  1236,\n",
              "  460,\n",
              "  272,\n",
              "  14391,\n",
              "  2280,\n",
              "  8807,\n",
              "  733,\n",
              "  28748,\n",
              "  16289,\n",
              "  28793,\n",
              "  3211,\n",
              "  752,\n",
              "  508,\n",
              "  28724,\n",
              "  28723,\n",
              "  18425,\n",
              "  653,\n",
              "  726,\n",
              "  26518,\n",
              "  13,\n",
              "  13,\n",
              "  1270,\n",
              "  13640,\n",
              "  28732,\n",
              "  28814,\n",
              "  1329,\n",
              "  13,\n",
              "  1318,\n",
              "  327,\n",
              "  1500,\n",
              "  28792,\n",
              "  28734,\n",
              "  28793,\n",
              "  13,\n",
              "  337,\n",
              "  327,\n",
              "  1500,\n",
              "  28792,\n",
              "  28740,\n",
              "  28793,\n",
              "  13,\n",
              "  604,\n",
              "  1318,\n",
              "  348,\n",
              "  28750,\n",
              "  648,\n",
              "  337,\n",
              "  348,\n",
              "  28750,\n",
              "  13,\n",
              "  13,\n",
              "  1270,\n",
              "  17656,\n",
              "  28740,\n",
              "  28732,\n",
              "  28814,\n",
              "  1329,\n",
              "  13,\n",
              "  1318,\n",
              "  327,\n",
              "  1500,\n",
              "  28792,\n",
              "  28734,\n",
              "  28793,\n",
              "  13,\n",
              "  337,\n",
              "  327,\n",
              "  1500,\n",
              "  28792,\n",
              "  28740,\n",
              "  28793,\n",
              "  13,\n",
              "  604,\n",
              "  1318,\n",
              "  648,\n",
              "  337,\n",
              "  387,\n",
              "  28705,\n",
              "  28740,\n",
              "  13,\n",
              "  13,\n",
              "  28814,\n",
              "  327,\n",
              "  325,\n",
              "  28740,\n",
              "  28723,\n",
              "  28734,\n",
              "  28725,\n",
              "  28740,\n",
              "  28723,\n",
              "  28734,\n",
              "  28731,\n",
              "  13,\n",
              "  28726,\n",
              "  327,\n",
              "  325,\n",
              "  28734,\n",
              "  28723,\n",
              "  28734,\n",
              "  28725,\n",
              "  5364,\n",
              "  28731,\n",
              "  13,\n",
              "  28726,\n",
              "  292,\n",
              "  28713,\n",
              "  327,\n",
              "  325,\n",
              "  28726,\n",
              "  28725,\n",
              "  28726,\n",
              "  28731,\n",
              "  13,\n",
              "  514,\n",
              "  28740,\n",
              "  327,\n",
              "  12012,\n",
              "  1123,\n",
              "  1869,\n",
              "  28742,\n",
              "  1673,\n",
              "  6078,\n",
              "  1755,\n",
              "  1869,\n",
              "  17656,\n",
              "  28740,\n",
              "  28752,\n",
              "  13,\n",
              "  28713,\n",
              "  2100,\n",
              "  327,\n",
              "  26518,\n",
              "  28732,\n",
              "  2814,\n",
              "  495,\n",
              "  28725,\n",
              "  1500,\n",
              "  28725,\n",
              "  2038,\n",
              "  2731,\n",
              "  6255,\n",
              "  28735,\n",
              "  28824,\n",
              "  28753,\n",
              "  647,\n",
              "  14594,\n",
              "  28746,\n",
              "  28726,\n",
              "  292,\n",
              "  28713,\n",
              "  28725,\n",
              "  14841,\n",
              "  28746,\n",
              "  514,\n",
              "  28740,\n",
              "  28731,\n",
              "  13,\n",
              "  13,\n",
              "  2031,\n",
              "  28732,\n",
              "  28713,\n",
              "  2100,\n",
              "  28731,\n",
              "  13,\n",
              "  28771,\n",
              "  15985,\n",
              "  28747,\n",
              "  28705,\n",
              "  746,\n",
              "  28747,\n",
              "  28705,\n",
              "  28734,\n",
              "  28723,\n",
              "  28774,\n",
              "  28774,\n",
              "  28774,\n",
              "  28774,\n",
              "  28774,\n",
              "  28774,\n",
              "  28774,\n",
              "  28774,\n",
              "  28774,\n",
              "  28774,\n",
              "  28750,\n",
              "  28774,\n",
              "  28781,\n",
              "  28774,\n",
              "  28770,\n",
              "  28784,\n",
              "  13,\n",
              "  2600,\n",
              "  1318,\n",
              "  28747,\n",
              "  733,\n",
              "  28734,\n",
              "  28723,\n",
              "  28781,\n",
              "  28774,\n",
              "  28774,\n",
              "  28774,\n",
              "  28774,\n",
              "  28774,\n",
              "  28774,\n",
              "  28774,\n",
              "  28705,\n",
              "  28750,\n",
              "  28723,\n",
              "  28781,\n",
              "  28774,\n",
              "  28774,\n",
              "  28774,\n",
              "  28774,\n",
              "  28774,\n",
              "  28774,\n",
              "  28774,\n",
              "  28793,\n",
              "  2],\n",
              " [1,\n",
              "  1,\n",
              "  733,\n",
              "  16289,\n",
              "  28793,\n",
              "  20548,\n",
              "  336,\n",
              "  349,\n",
              "  396,\n",
              "  13126,\n",
              "  369,\n",
              "  13966,\n",
              "  264,\n",
              "  3638,\n",
              "  28723,\n",
              "  12018,\n",
              "  264,\n",
              "  2899,\n",
              "  369,\n",
              "  6582,\n",
              "  1999,\n",
              "  2691,\n",
              "  274,\n",
              "  272,\n",
              "  2159,\n",
              "  28723,\n",
              "  13,\n",
              "  13,\n",
              "  26075,\n",
              "  264,\n",
              "  2696,\n",
              "  369,\n",
              "  25831,\n",
              "  575,\n",
              "  272,\n",
              "  11010,\n",
              "  302,\n",
              "  1430,\n",
              "  5498,\n",
              "  297,\n",
              "  272,\n",
              "  2078,\n",
              "  1423,\n",
              "  28723,\n",
              "  1236,\n",
              "  460,\n",
              "  272,\n",
              "  14391,\n",
              "  345,\n",
              "  6570,\n",
              "  2732,\n",
              "  2192,\n",
              "  5072,\n",
              "  28739,\n",
              "  733,\n",
              "  28748,\n",
              "  16289,\n",
              "  28793,\n",
              "  1270,\n",
              "  2682,\n",
              "  28730,\n",
              "  27902,\n",
              "  28732,\n",
              "  2186,\n",
              "  1329,\n",
              "  13,\n",
              "  2287,\n",
              "  586,\n",
              "  28730,\n",
              "  4063,\n",
              "  327,\n",
              "  4729,\n",
              "  13,\n",
              "  2287,\n",
              "  354,\n",
              "  5498,\n",
              "  297,\n",
              "  1707,\n",
              "  28747,\n",
              "  13,\n",
              "  5390,\n",
              "  513,\n",
              "  5498,\n",
              "  459,\n",
              "  297,\n",
              "  586,\n",
              "  28730,\n",
              "  4063,\n",
              "  28747,\n",
              "  13,\n",
              "  17422,\n",
              "  586,\n",
              "  28730,\n",
              "  4063,\n",
              "  28792,\n",
              "  15867,\n",
              "  28793,\n",
              "  327,\n",
              "  28705,\n",
              "  28740,\n",
              "  13,\n",
              "  5390,\n",
              "  1112,\n",
              "  28747,\n",
              "  13,\n",
              "  17422,\n",
              "  586,\n",
              "  28730,\n",
              "  4063,\n",
              "  28792,\n",
              "  15867,\n",
              "  28793,\n",
              "  2679,\n",
              "  28705,\n",
              "  28740,\n",
              "  13,\n",
              "  260,\n",
              "  13,\n",
              "  2287,\n",
              "  354,\n",
              "  5498,\n",
              "  28725,\n",
              "  285,\n",
              "  3644,\n",
              "  297,\n",
              "  586,\n",
              "  28730,\n",
              "  4063,\n",
              "  28723,\n",
              "  6949,\n",
              "  5888,\n",
              "  13,\n",
              "  5390,\n",
              "  2682,\n",
              "  618,\n",
              "  8779,\n",
              "  360,\n",
              "  28747,\n",
              "  16553,\n",
              "  15867,\n",
              "  28806,\n",
              "  548,\n",
              "  401,\n",
              "  17400,\n",
              "  28747,\n",
              "  16553,\n",
              "  1042,\n",
              "  28732,\n",
              "  14436,\n",
              "  743,\n",
              "  13,\n",
              "  260,\n",
              "  13,\n",
              "  335,\n",
              "  1848,\n",
              "  861,\n",
              "  860,\n",
              "  859,\n",
              "  464,\n",
              "  860,\n",
              "  5389,\n",
              "  860,\n",
              "  1869,\n",
              "  13,\n",
              "  2287,\n",
              "  2682,\n",
              "  28730,\n",
              "  27902,\n",
              "  618,\n",
              "  6570,\n",
              "  2732,\n",
              "  2192,\n",
              "  5072,\n",
              "  1243,\n",
              "  2],\n",
              " [1,\n",
              "  1,\n",
              "  733,\n",
              "  16289,\n",
              "  28793,\n",
              "  20548,\n",
              "  336,\n",
              "  349,\n",
              "  396,\n",
              "  13126,\n",
              "  369,\n",
              "  13966,\n",
              "  264,\n",
              "  3638,\n",
              "  28723,\n",
              "  12018,\n",
              "  264,\n",
              "  2899,\n",
              "  369,\n",
              "  6582,\n",
              "  1999,\n",
              "  2691,\n",
              "  274,\n",
              "  272,\n",
              "  2159,\n",
              "  28723,\n",
              "  13,\n",
              "  13,\n",
              "  8245,\n",
              "  264,\n",
              "  5599,\n",
              "  5168,\n",
              "  2229,\n",
              "  298,\n",
              "  875,\n",
              "  1575,\n",
              "  1581,\n",
              "  4514,\n",
              "  302,\n",
              "  10607,\n",
              "  2818,\n",
              "  356,\n",
              "  652,\n",
              "  11474,\n",
              "  28723,\n",
              "  733,\n",
              "  28748,\n",
              "  16289,\n",
              "  28793,\n",
              "  7841,\n",
              "  25637,\n",
              "  390,\n",
              "  7494,\n",
              "  13,\n",
              "  7841,\n",
              "  12140,\n",
              "  293,\n",
              "  390,\n",
              "  17479,\n",
              "  13,\n",
              "  7841,\n",
              "  1610,\n",
              "  11096,\n",
              "  4472,\n",
              "  28723,\n",
              "  2834,\n",
              "  11096,\n",
              "  390,\n",
              "  17768,\n",
              "  13,\n",
              "  13,\n",
              "  3211,\n",
              "  1321,\n",
              "  27082,\n",
              "  28723,\n",
              "  4201,\n",
              "  20363,\n",
              "  726,\n",
              "  3442,\n",
              "  28730,\n",
              "  28722,\n",
              "  551,\n",
              "  1046,\n",
              "  13,\n",
              "  3211,\n",
              "  1321,\n",
              "  27082,\n",
              "  28723,\n",
              "  3549,\n",
              "  28730,\n",
              "  15856,\n",
              "  726,\n",
              "  5835,\n",
              "  28730,\n",
              "  1613,\n",
              "  28730,\n",
              "  6220,\n",
              "  13,\n",
              "  13,\n",
              "  3211,\n",
              "  1321,\n",
              "  27082,\n",
              "  28723,\n",
              "  6243,\n",
              "  726,\n",
              "  6712,\n",
              "  1522,\n",
              "  7147,\n",
              "  2472,\n",
              "  3591,\n",
              "  13,\n",
              "  3211,\n",
              "  1321,\n",
              "  27082,\n",
              "  28723,\n",
              "  485,\n",
              "  956,\n",
              "  28726,\n",
              "  734,\n",
              "  726,\n",
              "  524,\n",
              "  6947,\n",
              "  956,\n",
              "  28726,\n",
              "  734,\n",
              "  2472,\n",
              "  3591,\n",
              "  13,\n",
              "  13,\n",
              "  28771,\n",
              "  11924,\n",
              "  272,\n",
              "  1178,\n",
              "  13,\n",
              "  28722,\n",
              "  551,\n",
              "  1046,\n",
              "  327,\n",
              "  3442,\n",
              "  28730,\n",
              "  28722,\n",
              "  551,\n",
              "  1046,\n",
              "  470,\n",
              "  13,\n",
              "  28814,\n",
              "  327,\n",
              "  21566,\n",
              "  28723,\n",
              "  1056,\n",
              "  13,\n",
              "  28724,\n",
              "  327,\n",
              "  21566,\n",
              "  28723,\n",
              "  3731,\n",
              "  13,\n",
              "  13,\n",
              "  28771,\n",
              "  318,\n",
              "  3255,\n",
              "  272,\n",
              "  1178,\n",
              "  778,\n",
              "  5835,\n",
              "  304,\n",
              "  1369,\n",
              "  6491,\n",
              "  13,\n",
              "  28814,\n",
              "  28730,\n",
              "  13641,\n",
              "  28725,\n",
              "  1500,\n",
              "  28730,\n",
              "  1613,\n",
              "  28725,\n",
              "  337,\n",
              "  28730,\n",
              "  13641,\n",
              "  28725,\n",
              "  337,\n",
              "  28730,\n",
              "  1613,\n",
              "  327,\n",
              "  5835,\n",
              "  28730,\n",
              "  1613,\n",
              "  28730,\n",
              "  6220,\n",
              "  28732,\n",
              "  28814,\n",
              "  28725,\n",
              "  337,\n",
              "  28725,\n",
              "  5509,\n",
              "  28730,\n",
              "  2027,\n",
              "  28746,\n",
              "  28734,\n",
              "  28731,\n",
              "  13,\n",
              "  13,\n",
              "  28771,\n",
              "  5670,\n",
              "  272,\n",
              "  875,\n",
              "  11980,\n",
              "  13,\n",
              "  6243,\n",
              "  327,\n",
              "  6712,\n",
              "  1522,\n",
              "  7147,\n",
              "  2472,\n",
              "  3591,\n",
              "  28732,\n",
              "  2416,\n",
              "  28730,\n",
              "  11685,\n",
              "  28746,\n",
              "  28781,\n",
              "  609,\n",
              "  7785,\n",
              "  28732,\n",
              "  28814,\n",
              "  28730,\n",
              "  13641,\n",
              "  28725,\n",
              "  337,\n",
              "  28730,\n",
              "  13641,\n",
              "  28731,\n",
              "  13,\n",
              "  28729,\n",
              "  9472,\n",
              "  327,\n",
              "  524,\n",
              "  6947,\n",
              "  956,\n",
              "  28726,\n",
              "  734,\n",
              "  2472,\n",
              "  3591,\n",
              "  28732,\n",
              "  28711,\n",
              "  28730,\n",
              "  485,\n",
              "  956,\n",
              "  28726,\n",
              "  734,\n",
              "  28746,\n",
              "  28782,\n",
              "  609,\n",
              "  7785,\n",
              "  28732,\n",
              "  28814,\n",
              "  28730,\n",
              "  13641,\n",
              "  28725,\n",
              "  337,\n",
              "  28730,\n",
              "  13641,\n",
              "  28731,\n",
              "  13,\n",
              "  13,\n",
              "  28771,\n",
              "  24223,\n",
              "  11931,\n",
              "  272,\n",
              "  4994,\n",
              "  13,\n",
              "  2031,\n",
              "  857,\n",
              "  6039,\n",
              "  1522,\n",
              "  14653,\n",
              "  28747,\n",
              "  371,\n",
              "  28747,\n",
              "  28723,\n",
              "  28750,\n",
              "  28722,\n",
              "  28752,\n",
              "  4135,\n",
              "  3762,\n",
              "  28732,\n",
              "  6243,\n",
              "  28723,\n",
              "  11831,\n",
              "  28732,\n",
              "  28814,\n",
              "  28730,\n",
              "  1613,\n",
              "  28725,\n",
              "  337,\n",
              "  28730,\n",
              "  1613,\n",
              "  5429,\n",
              "  13,\n",
              "  2031,\n",
              "  857,\n",
              "  28796,\n",
              "  11348,\n",
              "  28747,\n",
              "  371,\n",
              "  28747,\n",
              "  28723,\n",
              "  28750,\n",
              "  28722,\n",
              "  28752,\n",
              "  4135,\n",
              "  3762,\n",
              "  28732,\n",
              "  28729,\n",
              "  9472,\n",
              "  28723,\n",
              "  11831,\n",
              "  28732,\n",
              "  28814,\n",
              "  28730,\n",
              "  1613,\n",
              "  28725,\n",
              "  337,\n",
              "  28730,\n",
              "  1613,\n",
              "  5429,\n",
              "  28705,\n",
              "  2],\n",
              " [1,\n",
              "  1,\n",
              "  733,\n",
              "  16289,\n",
              "  28793,\n",
              "  20548,\n",
              "  336,\n",
              "  349,\n",
              "  396,\n",
              "  13126,\n",
              "  369,\n",
              "  13966,\n",
              "  264,\n",
              "  3638,\n",
              "  28723,\n",
              "  12018,\n",
              "  264,\n",
              "  2899,\n",
              "  369,\n",
              "  6582,\n",
              "  1999,\n",
              "  2691,\n",
              "  274,\n",
              "  272,\n",
              "  2159,\n",
              "  28723,\n",
              "  13,\n",
              "  13,\n",
              "  8587,\n",
              "  396,\n",
              "  9464,\n",
              "  354,\n",
              "  7484,\n",
              "  272,\n",
              "  22341,\n",
              "  304,\n",
              "  272,\n",
              "  7639,\n",
              "  5551,\n",
              "  297,\n",
              "  396,\n",
              "  521,\n",
              "  7653,\n",
              "  286,\n",
              "  2293,\n",
              "  302,\n",
              "  3113,\n",
              "  7850,\n",
              "  28723,\n",
              "  1236,\n",
              "  460,\n",
              "  272,\n",
              "  14391,\n",
              "  733,\n",
              "  28770,\n",
              "  28725,\n",
              "  28705,\n",
              "  28784,\n",
              "  28725,\n",
              "  28705,\n",
              "  28750,\n",
              "  28725,\n",
              "  28705,\n",
              "  28774,\n",
              "  28725,\n",
              "  28705,\n",
              "  28740,\n",
              "  28793,\n",
              "  733,\n",
              "  28748,\n",
              "  16289,\n",
              "  28793,\n",
              "  2001,\n",
              "  1062,\n",
              "  1066,\n",
              "  1187,\n",
              "  5010,\n",
              "  28732,\n",
              "  501,\n",
              "  2002,\n",
              "  4476,\n",
              "  28731,\n",
              "  371,\n",
              "  13,\n",
              "  716,\n",
              "  1187,\n",
              "  327,\n",
              "  4476,\n",
              "  28792,\n",
              "  28734,\n",
              "  1668,\n",
              "  13,\n",
              "  716,\n",
              "  2666,\n",
              "  327,\n",
              "  4476,\n",
              "  28792,\n",
              "  28734,\n",
              "  1668,\n",
              "  13,\n",
              "  13,\n",
              "  354,\n",
              "  325,\n",
              "  501,\n",
              "  613,\n",
              "  327,\n",
              "  28705,\n",
              "  28740,\n",
              "  28745,\n",
              "  613,\n",
              "  523,\n",
              "  4476,\n",
              "  28723,\n",
              "  2143,\n",
              "  28745,\n",
              "  613,\n",
              "  3073,\n",
              "  371,\n",
              "  13,\n",
              "  513,\n",
              "  325,\n",
              "  2654,\n",
              "  28792,\n",
              "  28710,\n",
              "  28793,\n",
              "  523,\n",
              "  1187,\n",
              "  28731,\n",
              "  371,\n",
              "  13,\n",
              "  1187,\n",
              "  327,\n",
              "  4476,\n",
              "  28792,\n",
              "  28710,\n",
              "  1668,\n",
              "  13,\n",
              "  443,\n",
              "  1112,\n",
              "  513,\n",
              "  325,\n",
              "  2654,\n",
              "  28792,\n",
              "  28710,\n",
              "  28793,\n",
              "  876,\n",
              "  2666,\n",
              "  28731,\n",
              "  371,\n",
              "  13,\n",
              "  2666,\n",
              "  327,\n",
              "  4476,\n",
              "  28792,\n",
              "  28710,\n",
              "  1668,\n",
              "  13,\n",
              "  443,\n",
              "  13,\n",
              "  443,\n",
              "  13,\n",
              "  2135,\n",
              "  28723,\n",
              "  406,\n",
              "  28723,\n",
              "  7561,\n",
              "  618,\n",
              "  5805,\n",
              "  327,\n",
              "  345,\n",
              "  648,\n",
              "  1187,\n",
              "  648,\n",
              "  7717,\n",
              "  5824,\n",
              "  327,\n",
              "  345,\n",
              "  648,\n",
              "  2666,\n",
              "  344,\n",
              "  13,\n",
              "  28752,\n",
              "  2],\n",
              " [1,\n",
              "  1,\n",
              "  733,\n",
              "  16289,\n",
              "  28793,\n",
              "  20548,\n",
              "  336,\n",
              "  349,\n",
              "  396,\n",
              "  13126,\n",
              "  369,\n",
              "  13966,\n",
              "  264,\n",
              "  3638,\n",
              "  28723,\n",
              "  12018,\n",
              "  264,\n",
              "  2899,\n",
              "  369,\n",
              "  6582,\n",
              "  1999,\n",
              "  2691,\n",
              "  274,\n",
              "  272,\n",
              "  2159,\n",
              "  28723,\n",
              "  13,\n",
              "  13,\n",
              "  5670,\n",
              "  264,\n",
              "  4686,\n",
              "  2884,\n",
              "  369,\n",
              "  4347,\n",
              "  989,\n",
              "  5551,\n",
              "  390,\n",
              "  396,\n",
              "  2787,\n",
              "  304,\n",
              "  6098,\n",
              "  272,\n",
              "  26281,\n",
              "  10977,\n",
              "  3060,\n",
              "  2038,\n",
              "  298,\n",
              "  4249,\n",
              "  272,\n",
              "  2648,\n",
              "  302,\n",
              "  272,\n",
              "  5551,\n",
              "  28723,\n",
              "  1236,\n",
              "  460,\n",
              "  272,\n",
              "  14391,\n",
              "  2280,\n",
              "  8807,\n",
              "  733,\n",
              "  28748,\n",
              "  16289,\n",
              "  14626,\n",
              "  28808,\n",
              "  5170,\n",
              "  1386,\n",
              "  12492,\n",
              "  12383,\n",
              "  28767,\n",
              "  13,\n",
              "  28789,\n",
              "  3391,\n",
              "  28767,\n",
              "  13,\n",
              "  28789,\n",
              "  1811,\n",
              "  28767,\n",
              "  13,\n",
              "  28705,\n",
              "  523,\n",
              "  3901,\n",
              "  28767,\n",
              "  9881,\n",
              "  2984,\n",
              "  2320,\n",
              "  1028,\n",
              "  700,\n",
              "  3901,\n",
              "  28767,\n",
              "  13,\n",
              "  523,\n",
              "  4459,\n",
              "  28767,\n",
              "  13,\n",
              "  28705,\n",
              "  908,\n",
              "  13911,\n",
              "  9881,\n",
              "  470,\n",
              "  371,\n",
              "  13,\n",
              "  2287,\n",
              "  779,\n",
              "  2075,\n",
              "  28740,\n",
              "  327,\n",
              "  3248,\n",
              "  28723,\n",
              "  14777,\n",
              "  618,\n",
              "  2575,\n",
              "  28740,\n",
              "  4145,\n",
              "  1431,\n",
              "  28745,\n",
              "  13,\n",
              "  2287,\n",
              "  779,\n",
              "  2075,\n",
              "  28750,\n",
              "  327,\n",
              "  3248,\n",
              "  28723,\n",
              "  14777,\n",
              "  618,\n",
              "  2575,\n",
              "  28750,\n",
              "  4145,\n",
              "  1431,\n",
              "  28745,\n",
              "  13,\n",
              "  2287,\n",
              "  779,\n",
              "  2648,\n",
              "  327,\n",
              "  26125,\n",
              "  28732,\n",
              "  2575,\n",
              "  28740,\n",
              "  28731,\n",
              "  648,\n",
              "  26125,\n",
              "  28732,\n",
              "  2575,\n",
              "  28750,\n",
              "  344,\n",
              "  13,\n",
              "  2287,\n",
              "  2924,\n",
              "  28723,\n",
              "  14762,\n",
              "  618,\n",
              "  9881,\n",
              "  349,\n",
              "  28747,\n",
              "  16553,\n",
              "  1801,\n",
              "  344,\n",
              "  13,\n",
              "  28705,\n",
              "  443,\n",
              "  13,\n",
              "  1867,\n",
              "  4459,\n",
              "  28767,\n",
              "  13,\n",
              "  700,\n",
              "  1811,\n",
              "  28767,\n",
              "  13,\n",
              "  28789,\n",
              "  2680,\n",
              "  28767,\n",
              "  13,\n",
              "  28705,\n",
              "  523,\n",
              "  28716,\n",
              "  28740,\n",
              "  28767,\n",
              "  9881,\n",
              "  2984,\n",
              "  2320,\n",
              "  1028,\n",
              "  700,\n",
              "  28716,\n",
              "  28740,\n",
              "  28767,\n",
              "  13,\n",
              "  28705,\n",
              "  523,\n",
              "  2537,\n",
              "  1212,\n",
              "  735,\n",
              "  772,\n",
              "  28739,\n",
              "  1910,\n",
              "  735,\n",
              "  2575,\n",
              "  28740,\n",
              "  28739,\n",
              "  17455,\n",
              "  735,\n",
              "  12403,\n",
              "  272,\n",
              "  907,\n",
              "  1474,\n",
              "  1355,\n",
              "  13,\n",
              "  28705,\n",
              "  523,\n",
              "  2537,\n",
              "  1212,\n",
              "  735,\n",
              "  772,\n",
              "  28739,\n",
              "  1910,\n",
              "  735,\n",
              "  2575,\n",
              "  28750,\n",
              "  28739,\n",
              "  17455,\n",
              "  735,\n",
              "  12403,\n",
              "  272,\n",
              "  1676,\n",
              "  1474,\n",
              "  1355,\n",
              "  13,\n",
              "  28705,\n",
              "  523,\n",
              "  4559,\n",
              "  28474,\n",
              "  735,\n",
              "  1391,\n",
              "  16914,\n",
              "  9881,\n",
              "  470,\n",
              "  1355,\n",
              "  7618,\n",
              "  16914,\n",
              "  6927,\n",
              "  700,\n",
              "  4559,\n",
              "  28767,\n",
              "  13,\n",
              "  700,\n",
              "  2680,\n",
              "  28767,\n",
              "  13,\n",
              "  700,\n",
              "  3391,\n",
              "  28767,\n",
              "  2],\n",
              " [1,\n",
              "  1,\n",
              "  733,\n",
              "  16289,\n",
              "  28793,\n",
              "  20548,\n",
              "  336,\n",
              "  349,\n",
              "  396,\n",
              "  13126,\n",
              "  369,\n",
              "  13966,\n",
              "  264,\n",
              "  3638,\n",
              "  28723,\n",
              "  12018,\n",
              "  264,\n",
              "  2899,\n",
              "  369,\n",
              "  6582,\n",
              "  1999,\n",
              "  2691,\n",
              "  274,\n",
              "  272,\n",
              "  2159,\n",
              "  28723,\n",
              "  13,\n",
              "  13,\n",
              "  26075,\n",
              "  264,\n",
              "  21966,\n",
              "  2007,\n",
              "  369,\n",
              "  26223,\n",
              "  989,\n",
              "  5551,\n",
              "  304,\n",
              "  25831,\n",
              "  652,\n",
              "  4518,\n",
              "  28723,\n",
              "  733,\n",
              "  28748,\n",
              "  16289,\n",
              "  28793,\n",
              "  1270,\n",
              "  967,\n",
              "  28732,\n",
              "  28744,\n",
              "  28725,\n",
              "  337,\n",
              "  1329,\n",
              "  13,\n",
              "  2287,\n",
              "  2682,\n",
              "  28732,\n",
              "  28744,\n",
              "  648,\n",
              "  337,\n",
              "  28731,\n",
              "  13,\n",
              "  13,\n",
              "  28708,\n",
              "  327,\n",
              "  716,\n",
              "  28732,\n",
              "  2537,\n",
              "  618,\n",
              "  12403,\n",
              "  907,\n",
              "  1474,\n",
              "  28747,\n",
              "  345,\n",
              "  743,\n",
              "  13,\n",
              "  28726,\n",
              "  327,\n",
              "  716,\n",
              "  28732,\n",
              "  2537,\n",
              "  618,\n",
              "  12403,\n",
              "  1676,\n",
              "  1474,\n",
              "  28747,\n",
              "  345,\n",
              "  743,\n",
              "  13,\n",
              "  988,\n",
              "  28732,\n",
              "  28708,\n",
              "  28725,\n",
              "  287,\n",
              "  28731,\n",
              "  28705,\n",
              "  2],\n",
              " [1,\n",
              "  1,\n",
              "  733,\n",
              "  16289,\n",
              "  28793,\n",
              "  20548,\n",
              "  336,\n",
              "  349,\n",
              "  396,\n",
              "  13126,\n",
              "  369,\n",
              "  13966,\n",
              "  264,\n",
              "  3638,\n",
              "  28723,\n",
              "  12018,\n",
              "  264,\n",
              "  2899,\n",
              "  369,\n",
              "  6582,\n",
              "  1999,\n",
              "  2691,\n",
              "  274,\n",
              "  272,\n",
              "  2159,\n",
              "  28723,\n",
              "  13,\n",
              "  13,\n",
              "  12018,\n",
              "  396,\n",
              "  13208,\n",
              "  5709,\n",
              "  298,\n",
              "  1300,\n",
              "  272,\n",
              "  2955,\n",
              "  302,\n",
              "  544,\n",
              "  272,\n",
              "  7896,\n",
              "  693,\n",
              "  460,\n",
              "  2739,\n",
              "  297,\n",
              "  272,\n",
              "  1348,\n",
              "  9784,\n",
              "  390,\n",
              "  2215,\n",
              "  28723,\n",
              "  1236,\n",
              "  460,\n",
              "  272,\n",
              "  14391,\n",
              "  7582,\n",
              "  345,\n",
              "  3729,\n",
              "  16199,\n",
              "  28739,\n",
              "  13,\n",
              "  13,\n",
              "  28766,\n",
              "  6318,\n",
              "  342,\n",
              "  6620,\n",
              "  28705,\n",
              "  342,\n",
              "  6946,\n",
              "  342,\n",
              "  13,\n",
              "  28766,\n",
              "  502,\n",
              "  28766,\n",
              "  25702,\n",
              "  28766,\n",
              "  9762,\n",
              "  28766,\n",
              "  13,\n",
              "  28766,\n",
              "  28705,\n",
              "  28740,\n",
              "  28705,\n",
              "  342,\n",
              "  2215,\n",
              "  28705,\n",
              "  342,\n",
              "  21664,\n",
              "  273,\n",
              "  342,\n",
              "  13,\n",
              "  28766,\n",
              "  28705,\n",
              "  28750,\n",
              "  28705,\n",
              "  342,\n",
              "  4299,\n",
              "  28705,\n",
              "  342,\n",
              "  21664,\n",
              "  273,\n",
              "  342,\n",
              "  13,\n",
              "  28766,\n",
              "  28705,\n",
              "  28770,\n",
              "  28705,\n",
              "  342,\n",
              "  14003,\n",
              "  342,\n",
              "  8862,\n",
              "  273,\n",
              "  342,\n",
              "  13,\n",
              "  28766,\n",
              "  28705,\n",
              "  28781,\n",
              "  28705,\n",
              "  342,\n",
              "  7409,\n",
              "  259,\n",
              "  342,\n",
              "  21664,\n",
              "  273,\n",
              "  342,\n",
              "  733,\n",
              "  28748,\n",
              "  16289,\n",
              "  28793,\n",
              "  10696,\n",
              "  6620,\n",
              "  28705,\n",
              "  13,\n",
              "  21335,\n",
              "  2929,\n",
              "  16199,\n",
              "  28705,\n",
              "  13,\n",
              "  28780,\n",
              "  11724,\n",
              "  6946,\n",
              "  327,\n",
              "  325,\n",
              "  10696,\n",
              "  6946,\n",
              "  10657,\n",
              "  2929,\n",
              "  16199,\n",
              "  15803,\n",
              "  6620,\n",
              "  327,\n",
              "  464,\n",
              "  14964,\n",
              "  2207,\n",
              "  2]]"
            ]
          },
          "execution_count": 26,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "train_data[\"input_ids\"][:10]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### After Formatting, We should get something like this\n",
        "\n",
        "```json\n",
        "{\n",
        "\"text\":\"<s>[INST] Create a function to calculate the sum of a sequence of integers. here are the inputs [1, 2, 3, 4, 5] [/INST]\n",
        "# Python code def sum_sequence(sequence): sum = 0 for num in sequence: sum += num return sum</s>\",\n",
        "\"instruction\":\"Create a function to calculate the sum of a sequence of integers\",\n",
        "\"input\":\"[1, 2, 3, 4, 5]\",\n",
        "\"output\":\"# Python code def sum_sequence(sequence): sum = 0 for num in,\n",
        " sequence: sum += num return sum\"\n",
        "\"prompt\":\"<s>[INST] Create a function to calculate the sum of a sequence of integers. here are the inputs [1, 2, 3, 4, 5] [/INST]\n",
        "# Python code def sum_sequence(sequence): sum = 0 for num in sequence: sum += num return sum</s>\"\n",
        "\n",
        "}\n",
        "```\n",
        "\n",
        "While using SFT (**[Supervised Fine-tuning Trainer](https://huggingface.co/docs/trl/main/en/sft_trainer)**) for fine-tuning, we will be only passing in the “text” column of the dataset for fine-tuning."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Dataset({\n",
            "    features: ['instruction', 'output', 'input', 'text', 'prompt', 'input_ids', 'attention_mask'],\n",
            "    num_rows: 24392\n",
            "})\n"
          ]
        }
      ],
      "source": [
        "print(test_data)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g-Vs5Dc3DdY_"
      },
      "source": [
        "### Setting up the Training\n",
        "we will be using the `huggingface` and the `peft` library!"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "id": "SQulqDzjd0gD"
      },
      "outputs": [],
      "source": [
        "from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training\n",
        "\n",
        "peft_config = LoraConfig(\n",
        "    lora_alpha=16,\n",
        "    lora_dropout=0.1,\n",
        "    r=64,\n",
        "    bias=\"none\",\n",
        "        target_modules=[\n",
        "        \"q_proj\",\n",
        "        \"k_proj\",\n",
        "        \"v_proj\",\n",
        "        \"o_proj\",\n",
        "        \"gate_proj\",\n",
        "        \"up_proj\",\n",
        "        \"down_proj\",\n",
        "        \"lm_head\",\n",
        "    ],\n",
        "    task_type=\"CAUSAL_LM\"\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1SWl1MgjDrXX"
      },
      "source": [
        "we need to prepare the model to be trained in 4bit so we will use the  `prepare_model_for_kbit_training` function from peft\n",
        "\n",
        "> Indented block\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {
        "id": "bcco3ITVd486"
      },
      "outputs": [],
      "source": [
        "model = prepare_model_for_kbit_training(model)\n",
        "model = get_peft_model(model, peft_config)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "metadata": {
        "id": "BJzV4BRhyVQU"
      },
      "outputs": [],
      "source": [
        "def print_trainable_parameters(model):\n",
        "    \"\"\"\n",
        "    Prints the number of trainable parameters in the model.\n",
        "    \"\"\"\n",
        "    trainable_params = 0\n",
        "    all_param = 0\n",
        "    for _, param in model.named_parameters():\n",
        "        all_param += param.numel()\n",
        "        if param.requires_grad:\n",
        "            trainable_params += param.numel()\n",
        "    print(\n",
        "        f\"trainable params: {trainable_params} || all params: {all_param} || trainable%: {100 * trainable_params / all_param}\"\n",
        "    )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 31,
      "metadata": {
        "id": "aWqhs0W-yVQk",
        "outputId": "29093942-dc74-43e8-fa29-e5177f7c75cb"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "trainable params: 56836096 || all params: 23539437568 || trainable%: 0.24145052674182907\n"
          ]
        }
      ],
      "source": [
        "print_trainable_parameters(model)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Model after Adding Lora Config"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 32,
      "metadata": {
        "id": "lxN2-hyHyVQk",
        "outputId": "342ea747-938a-44c3-cad5-b94b79ebcf0b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "PeftModelForCausalLM(\n",
            "  (base_model): LoraModel(\n",
            "    (model): MixtralForCausalLM(\n",
            "      (model): MixtralModel(\n",
            "        (embed_tokens): Embedding(32000, 4096)\n",
            "        (layers): ModuleList(\n",
            "          (0-31): 32 x MixtralDecoderLayer(\n",
            "            (self_attn): MixtralFlashAttention2(\n",
            "              (q_proj): lora.Linear4bit(\n",
            "                (base_layer): Linear4bit(in_features=4096, out_features=4096, bias=False)\n",
            "                (lora_dropout): ModuleDict(\n",
            "                  (default): Dropout(p=0.1, inplace=False)\n",
            "                )\n",
            "                (lora_A): ModuleDict(\n",
            "                  (default): Linear(in_features=4096, out_features=64, bias=False)\n",
            "                )\n",
            "                (lora_B): ModuleDict(\n",
            "                  (default): Linear(in_features=64, out_features=4096, bias=False)\n",
            "                )\n",
            "                (lora_embedding_A): ParameterDict()\n",
            "                (lora_embedding_B): ParameterDict()\n",
            "              )\n",
            "              (k_proj): lora.Linear4bit(\n",
            "                (base_layer): Linear4bit(in_features=4096, out_features=1024, bias=False)\n",
            "                (lora_dropout): ModuleDict(\n",
            "                  (default): Dropout(p=0.1, inplace=False)\n",
            "                )\n",
            "                (lora_A): ModuleDict(\n",
            "                  (default): Linear(in_features=4096, out_features=64, bias=False)\n",
            "                )\n",
            "                (lora_B): ModuleDict(\n",
            "                  (default): Linear(in_features=64, out_features=1024, bias=False)\n",
            "                )\n",
            "                (lora_embedding_A): ParameterDict()\n",
            "                (lora_embedding_B): ParameterDict()\n",
            "              )\n",
            "              (v_proj): lora.Linear4bit(\n",
            "                (base_layer): Linear4bit(in_features=4096, out_features=1024, bias=False)\n",
            "                (lora_dropout): ModuleDict(\n",
            "                  (default): Dropout(p=0.1, inplace=False)\n",
            "                )\n",
            "                (lora_A): ModuleDict(\n",
            "                  (default): Linear(in_features=4096, out_features=64, bias=False)\n",
            "                )\n",
            "                (lora_B): ModuleDict(\n",
            "                  (default): Linear(in_features=64, out_features=1024, bias=False)\n",
            "                )\n",
            "                (lora_embedding_A): ParameterDict()\n",
            "                (lora_embedding_B): ParameterDict()\n",
            "              )\n",
            "              (o_proj): lora.Linear4bit(\n",
            "                (base_layer): Linear4bit(in_features=4096, out_features=4096, bias=False)\n",
            "                (lora_dropout): ModuleDict(\n",
            "                  (default): Dropout(p=0.1, inplace=False)\n",
            "                )\n",
            "                (lora_A): ModuleDict(\n",
            "                  (default): Linear(in_features=4096, out_features=64, bias=False)\n",
            "                )\n",
            "                (lora_B): ModuleDict(\n",
            "                  (default): Linear(in_features=64, out_features=4096, bias=False)\n",
            "                )\n",
            "                (lora_embedding_A): ParameterDict()\n",
            "                (lora_embedding_B): ParameterDict()\n",
            "              )\n",
            "              (rotary_emb): MixtralRotaryEmbedding()\n",
            "            )\n",
            "            (block_sparse_moe): MixtralSparseMoeBlock(\n",
            "              (gate): Linear4bit(in_features=4096, out_features=8, bias=False)\n",
            "              (experts): ModuleList(\n",
            "                (0-7): 8 x MixtralBLockSparseTop2MLP(\n",
            "                  (w1): Linear4bit(in_features=4096, out_features=14336, bias=False)\n",
            "                  (w2): Linear4bit(in_features=14336, out_features=4096, bias=False)\n",
            "                  (w3): Linear4bit(in_features=4096, out_features=14336, bias=False)\n",
            "                  (act_fn): SiLU()\n",
            "                )\n",
            "              )\n",
            "            )\n",
            "            (input_layernorm): MixtralRMSNorm()\n",
            "            (post_attention_layernorm): MixtralRMSNorm()\n",
            "          )\n",
            "        )\n",
            "        (norm): MixtralRMSNorm()\n",
            "      )\n",
            "      (lm_head): lora.Linear(\n",
            "        (base_layer): Linear(in_features=4096, out_features=32000, bias=False)\n",
            "        (lora_dropout): ModuleDict(\n",
            "          (default): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (lora_A): ModuleDict(\n",
            "          (default): Linear(in_features=4096, out_features=64, bias=False)\n",
            "        )\n",
            "        (lora_B): ModuleDict(\n",
            "          (default): Linear(in_features=64, out_features=32000, bias=False)\n",
            "        )\n",
            "        (lora_embedding_A): ParameterDict()\n",
            "        (lora_embedding_B): ParameterDict()\n",
            "      )\n",
            "    )\n",
            "  )\n",
            ")\n"
          ]
        }
      ],
      "source": [
        "print(model)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "go8_nVq7jO9d"
      },
      "source": [
        "### Hyper-paramters for training\n",
        "These parameters will depend on how long you want to run training for.\n",
        "Most important to consider:\n",
        "\n",
        "`num_train_epochs/max_steps`: How many iterations over the data you want to do, BE CAREFUL, don't try too many, you will over-fit!!!!!\n",
        "\n",
        "`learning_rate`: Controls the speed of convergence\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 33,
      "metadata": {
        "id": "hdzkKPXvyVQk",
        "outputId": "2b645f4e-673b-49ec-d463-94a0f806adfe"
      },
      "outputs": [],
      "source": [
        "if torch.cuda.device_count() > 1: # If more than 1 GPU\n",
        "    print(torch.cuda.device_count())\n",
        "    model.is_parallelizable = True\n",
        "    model.model_parallel = True"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 36,
      "metadata": {
        "id": "YhoCjs9md8pB"
      },
      "outputs": [],
      "source": [
        "from transformers import TrainingArguments\n",
        "\n",
        "args = TrainingArguments(\n",
        "  output_dir = \"Mixtral_Alpace_v3\",\n",
        "  #num_train_epochs=5,\n",
        "  max_steps = 100, # comment out this line if you want to train in epochs\n",
        "  per_device_train_batch_size = 32,\n",
        "  warmup_steps = 0.03,\n",
        "  logging_steps=10,\n",
        "  save_strategy=\"epoch\",\n",
        "  #evaluation_strategy=\"epoch\",\n",
        "  evaluation_strategy=\"steps\",\n",
        "  eval_steps=10, # comment out this line if you want to evaluate at the end of each epoch\n",
        "  learning_rate=2.5e-5,\n",
        "  bf16=True,\n",
        "  # lr_scheduler_type='constant',\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Gz6Vhh4WFpMM"
      },
      "source": [
        "Setting up the trainer.\n",
        "\n",
        "`max_seq_length`: Context window size\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 39,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UyyNtDrmeAkc",
        "outputId": "4e992c2f-38eb-40fd-e9cf-d4f8ec301b76"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4c9baed51ece4f4fa74ea806de1fb5a5",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Generating train split: 0 examples [00:00, ? examples/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d11c020de54a47548665d5d80398e7b4",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Generating train split: 0 examples [00:00, ? examples/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/home/adithya/miniconda3/envs/train-venv/lib/python3.10/site-packages/trl/trainer/sft_trainer.py:302: UserWarning: You passed `packing=True` to the SFTTrainer, and you are training your model with `max_steps` strategy. The dataset will be iterated until the `max_steps` are reached.\n",
            "  warnings.warn(\n"
          ]
        }
      ],
      "source": [
        "from trl import SFTTrainer\n",
        "\n",
        "max_seq_length = 1024\n",
        "\n",
        "trainer = SFTTrainer(\n",
        "  model=model,\n",
        "  peft_config=peft_config,\n",
        "  max_seq_length=max_seq_length,\n",
        "  tokenizer=tokenizer,\n",
        "  packing=True,\n",
        "  args=args,\n",
        "  dataset_text_field=\"prompt\",\n",
        "  train_dataset=train_data,\n",
        "  eval_dataset=test_data,\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 40,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 373
        },
        "id": "bsOO4bR9fQBb",
        "outputId": "cb946242-b113-4a57-a70b-8fbe05bbac07"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/home/adithya/miniconda3/envs/train-venv/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
            "  warnings.warn(\n",
            "The input hidden states seems to be silently casted in float32, this might be related to the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in torch.bfloat16.\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='11' max='100' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [ 11/100 03:06 < 30:42, 0.05 it/s, Epoch 0.02/1]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "  </tbody>\n",
              "</table><p>\n",
              "    <div>\n",
              "      \n",
              "      <progress value='606' max='642' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [606/642 20:52 < 01:14, 0.48 it/s]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "trainer.train()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ysHgWnwbfRSt"
      },
      "outputs": [],
      "source": [
        "trainer.save_model(\"Mixtral_Alpace_v2\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "na6xC6f-mGGm"
      },
      "source": [
        "# Save Model and Push to Hub"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SBPVlNe0j-Nk"
      },
      "outputs": [],
      "source": [
        "# !pip install huggingface-hub -qU"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uCDmkWCXnmuv"
      },
      "outputs": [],
      "source": [
        "# from huggingface_hub import notebook_login\n",
        "\n",
        "# notebook_login()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HJTZaSXqnqPa"
      },
      "outputs": [],
      "source": [
        "# trainer.push_to_hub(\"Promptengineering/mistral-instruct-generation\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9zYrr6sfkA6M"
      },
      "outputs": [],
      "source": [
        "merged_model = model.merge_and_unload()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "c1Eo6D2mnOgN"
      },
      "outputs": [],
      "source": [
        "def generate_response(prompt, model):\n",
        "  encoded_input = tokenizer(prompt,  return_tensors=\"pt\", add_special_tokens=True)\n",
        "  model_inputs = encoded_input.to('cuda')\n",
        "\n",
        "  generated_ids = model.generate(**model_inputs,\n",
        "                                 max_new_tokens=150,\n",
        "                                 do_sample=True,\n",
        "                                 pad_token_id=tokenizer.eos_token_id)\n",
        "\n",
        "  decoded_output = tokenizer.batch_decode(generated_ids)\n",
        "\n",
        "  return decoded_output[0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fVhc6eD-yVQl"
      },
      "outputs": [],
      "source": [
        "prompt = \"[INST]Use the provided input to create an instruction that could have been used to generate the response with an LLM.\\nThere are more than 12,000 species of grass. The most common is Kentucky Bluegrass, because it grows quickly, easily, and is soft to the touch. Rygrass is shiny and bright green colored. Fescues are dark green and shiny. Bermuda grass is harder but can grow in drier soil.[/INST]\"\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NMFDDMwly9L4"
      },
      "outputs": [],
      "source": [
        "generate_response(prompt, merged_model)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ldSnbui8yVQl",
        "outputId": "48427c76-a18c-47b8-b92b-d54e8cf67a60"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "8000"
            ]
          },
          "execution_count": 51,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "250*32"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "V100",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.13"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "0055eb4d0d054684878bba6da9cae910": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "084223b1556a420da5964ee086c117e0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "09f69e88c4834d25be2be790022384b0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0b9648d0eccf4516a8c0a7e3843f76b8": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "118879d64cae46f5b3e744f86404922c": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "164bc349194f4e6ab1229f3d2d03b3db": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1c03379c0bf84b62a671021810a5caee": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2073bd16554341df9ea62874f4907e18": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "21ac78e940c84842bdde1b64bd1395ad": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "240f5b4175e4469381a53eacdf657419": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "290face1ed4647d7b172109e162c3769": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_773040f77dae4115aefdd958a658a3fc",
            "max": 116,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_876619b7e19441a096225f72e58b5544",
            "value": 116
          }
        },
        "296ee6c082da4c76be2b099484662d26": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2998cf3d58354f6f8465a6e2a10ebab2": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2de61bee8b56442887b6f42592423064": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2efa53a79dd54d86b72284f3a87c7ca9": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_164bc349194f4e6ab1229f3d2d03b3db",
            "placeholder": "​",
            "style": "IPY_MODEL_732e5508dfb1422c97c0fbc27b888f91",
            "value": " 571/571 [00:00&lt;00:00, 36.4kB/s]"
          }
        },
        "3454ad0be8554f39bea55eb8d9c3db1b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "35f2eadd1bca48bdbc6504fcd693dab7": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4245f82dafc0478ca7ac1f5b372e5a12": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_82c83b75261c4f1db9b7c3d9ad0d097e",
            "placeholder": "​",
            "style": "IPY_MODEL_fc38a40ba92d46c49b3679a84c8ef879",
            "value": "pytorch_model-00002-of-00002.bin: 100%"
          }
        },
        "42becf797d9a42bc9984d8e30b307cff": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "444b9f99a79045b9a5c779a3d18bedd4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_240f5b4175e4469381a53eacdf657419",
            "placeholder": "​",
            "style": "IPY_MODEL_7af0d44687f142288e3e58f15338dedd",
            "value": "generation_config.json: 100%"
          }
        },
        "48648a0d46104a0db77eabf6d8fdf0f0": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "494347c07d684876ba64b39d698250d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fb5aaa2a96e047a0aa47d07a07b183fe",
            "max": 5064823659,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f7a940cb871f41c7af7e79c32c3156e7",
            "value": 5064823659
          }
        },
        "4fbd859668ac43c884d39f675f1158f7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fbe68ed8ac6243f180408175284b98f5",
            "placeholder": "​",
            "style": "IPY_MODEL_86712973ec7e498da66f29dbcea3f24f",
            "value": "config.json: 100%"
          }
        },
        "50b0bd7f60ea41c59d6cbd055a10d765": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_85f25707d0a04084aaa1ada15616f415",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_553955d94d844f05bc19116611761127",
            "value": 2
          }
        },
        "50d301095b594acdb2d754377b029245": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "553955d94d844f05bc19116611761127": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6223c94b2b814f2ab7abe976402551b1": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f56251c2b16c4dae99f80db60e5f6330",
            "placeholder": "​",
            "style": "IPY_MODEL_9e98f0e4ef964bbeaa63ca278313bb19",
            "value": " 5.06G/5.06G [00:17&lt;00:00, 438MB/s]"
          }
        },
        "643f193c688d49a0a204ea70019f2cc5": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "68b12f6f28fa452ea302d83e37084c43": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f369a81a80d84934881c8e9a0bc6a14f",
            "max": 571,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_72588505d177495b88c39917ae9f743c",
            "value": 571
          }
        },
        "6950d59bdf254dd896c9f30a220c0533": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0b9648d0eccf4516a8c0a7e3843f76b8",
            "max": 23950,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a2a870bddb694a8ea69bdac062ef6d46",
            "value": 23950
          }
        },
        "72588505d177495b88c39917ae9f743c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "72a4b9321abf4d58a093a3d09263c1ad": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "732e5508dfb1422c97c0fbc27b888f91": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "74b57be114494edda79768ff04961193": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "773040f77dae4115aefdd958a658a3fc": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "79d82f5e0881485897a8e977269079c8": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "7af0d44687f142288e3e58f15338dedd": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7d8f03389fc64ddfa128be53ea2eb1e8": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_21ac78e940c84842bdde1b64bd1395ad",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ddfc0a4121b8447b84abab68482e46a3",
            "value": 2
          }
        },
        "7e5645b33aa64e7ebbd60cb76e1ecd11": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_4245f82dafc0478ca7ac1f5b372e5a12",
              "IPY_MODEL_494347c07d684876ba64b39d698250d0",
              "IPY_MODEL_6223c94b2b814f2ab7abe976402551b1"
            ],
            "layout": "IPY_MODEL_48648a0d46104a0db77eabf6d8fdf0f0"
          }
        },
        "82c83b75261c4f1db9b7c3d9ad0d097e": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "82cafea491ca4350bea831165926d9fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f3d9d131f5934a048930bc8983437983",
              "IPY_MODEL_50b0bd7f60ea41c59d6cbd055a10d765",
              "IPY_MODEL_f27e89855a374c6d93e292dce8e146c5"
            ],
            "layout": "IPY_MODEL_72a4b9321abf4d58a093a3d09263c1ad"
          }
        },
        "838cd2c0701c4b0b82fa8ddce51215db": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "85f25707d0a04084aaa1ada15616f415": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "86712973ec7e498da66f29dbcea3f24f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "876619b7e19441a096225f72e58b5544": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8aec8536c28f4c75b2728936f6843a2c": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "90de7d165e804fbf8f3782ac5b16f8fb": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "95863a2d3c074f45b79a894cb8e67d27": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "963c8c0504554e57bb9a6a8e2c2060fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_118879d64cae46f5b3e744f86404922c",
            "placeholder": "​",
            "style": "IPY_MODEL_0055eb4d0d054684878bba6da9cae910",
            "value": "pytorch_model.bin.index.json: 100%"
          }
        },
        "9e98f0e4ef964bbeaa63ca278313bb19": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a2a870bddb694a8ea69bdac062ef6d46": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "adcfd8c2daa14f81aea6c85bf98cb735": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_42becf797d9a42bc9984d8e30b307cff",
            "placeholder": "​",
            "style": "IPY_MODEL_1c03379c0bf84b62a671021810a5caee",
            "value": "Downloading shards: 100%"
          }
        },
        "ae012e86315f41e7ae12b84312d2455b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_444b9f99a79045b9a5c779a3d18bedd4",
              "IPY_MODEL_290face1ed4647d7b172109e162c3769",
              "IPY_MODEL_fbeb4393ccb34413bea869355235d0fa"
            ],
            "layout": "IPY_MODEL_838cd2c0701c4b0b82fa8ddce51215db"
          }
        },
        "b245d2f284d7450b9d99e820f6193739": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b3a9937a7f5642e1bdddd2cb1c723490": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2de61bee8b56442887b6f42592423064",
            "max": 9943028044,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_79d82f5e0881485897a8e977269079c8",
            "value": 9943028044
          }
        },
        "b414aed96db744e99cbe4bf052426fa9": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "becf709cc6a4490481f6cbebbc6bad49": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_df28571a454d47d698d56d7cf4d90009",
            "placeholder": "​",
            "style": "IPY_MODEL_b414aed96db744e99cbe4bf052426fa9",
            "value": " 23.9k/23.9k [00:00&lt;00:00, 1.83MB/s]"
          }
        },
        "c07ef3e83cec46ef8a9fc9d7d3fdb6fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b245d2f284d7450b9d99e820f6193739",
            "placeholder": "​",
            "style": "IPY_MODEL_09f69e88c4834d25be2be790022384b0",
            "value": " 9.94G/9.94G [00:32&lt;00:00, 324MB/s]"
          }
        },
        "c577ab91428c4f5aac8299ba85104dc6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_4fbd859668ac43c884d39f675f1158f7",
              "IPY_MODEL_68b12f6f28fa452ea302d83e37084c43",
              "IPY_MODEL_2efa53a79dd54d86b72284f3a87c7ca9"
            ],
            "layout": "IPY_MODEL_643f193c688d49a0a204ea70019f2cc5"
          }
        },
        "c65ab63b72d143c7bca446276d0f1aa5": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_963c8c0504554e57bb9a6a8e2c2060fe",
              "IPY_MODEL_6950d59bdf254dd896c9f30a220c0533",
              "IPY_MODEL_becf709cc6a4490481f6cbebbc6bad49"
            ],
            "layout": "IPY_MODEL_90de7d165e804fbf8f3782ac5b16f8fb"
          }
        },
        "ccbc22e41b754682bea5e80531498470": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d429cb7700aa40099e28fa9b6565be26": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_db518d1bca324c98911ad35e5df91c3a",
              "IPY_MODEL_b3a9937a7f5642e1bdddd2cb1c723490",
              "IPY_MODEL_c07ef3e83cec46ef8a9fc9d7d3fdb6fe"
            ],
            "layout": "IPY_MODEL_74b57be114494edda79768ff04961193"
          }
        },
        "dae5a71abe874e69b3972d5fc861fd1a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "db518d1bca324c98911ad35e5df91c3a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ccbc22e41b754682bea5e80531498470",
            "placeholder": "​",
            "style": "IPY_MODEL_3454ad0be8554f39bea55eb8d9c3db1b",
            "value": "pytorch_model-00001-of-00002.bin: 100%"
          }
        },
        "ddfc0a4121b8447b84abab68482e46a3": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "df28571a454d47d698d56d7cf4d90009": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ed774deaa35441578397fa1241118263": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_adcfd8c2daa14f81aea6c85bf98cb735",
              "IPY_MODEL_7d8f03389fc64ddfa128be53ea2eb1e8",
              "IPY_MODEL_fd169008a90840f69ca3ff57ad7f84ed"
            ],
            "layout": "IPY_MODEL_8aec8536c28f4c75b2728936f6843a2c"
          }
        },
        "f27e89855a374c6d93e292dce8e146c5": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_296ee6c082da4c76be2b099484662d26",
            "placeholder": "​",
            "style": "IPY_MODEL_2073bd16554341df9ea62874f4907e18",
            "value": " 2/2 [00:14&lt;00:00,  6.73s/it]"
          }
        },
        "f369a81a80d84934881c8e9a0bc6a14f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f3d9d131f5934a048930bc8983437983": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_35f2eadd1bca48bdbc6504fcd693dab7",
            "placeholder": "​",
            "style": "IPY_MODEL_084223b1556a420da5964ee086c117e0",
            "value": "Loading checkpoint shards: 100%"
          }
        },
        "f56251c2b16c4dae99f80db60e5f6330": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f7a940cb871f41c7af7e79c32c3156e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "fb5aaa2a96e047a0aa47d07a07b183fe": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fbe68ed8ac6243f180408175284b98f5": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fbeb4393ccb34413bea869355235d0fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_95863a2d3c074f45b79a894cb8e67d27",
            "placeholder": "​",
            "style": "IPY_MODEL_2998cf3d58354f6f8465a6e2a10ebab2",
            "value": " 116/116 [00:00&lt;00:00, 9.67kB/s]"
          }
        },
        "fc38a40ba92d46c49b3679a84c8ef879": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fd169008a90840f69ca3ff57ad7f84ed": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_50d301095b594acdb2d754377b029245",
            "placeholder": "​",
            "style": "IPY_MODEL_dae5a71abe874e69b3972d5fc861fd1a",
            "value": " 2/2 [00:51&lt;00:00, 24.32s/it]"
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}
